import DataProcess
import Core.Gadget as Gadget
from Core.Config import *
config = Config()
database = config.DataBase()

import math
import DataProcess
import pandas as pd
import datetime
import numpy as np
import Core.WindFunctions as Wind


# opprofit                  #Missing 82     #None 0     #Zero 20
# tot_oper_rev              #Missing 82     #None 0     #Zero 335
# net_invest_inc            #Missing 82     #None 921   #Zero 26367
# other_grants_inc          #Missing 0      #None 79404 #Zero 1958
# net_gain_chg_fv           #Missing 92963  #None 921   #Zero 2021
# gain_asset_dispositions   #Missing 0      #None 79404 #Zero 4954
# net_gain_fx_trans         #Missing 92963  #None 921   #Zero 2967
# net_other_oper_inc

def CheckNullField(fields):
    #
    instruments = database.Find("Instruments", "Stock")
    instrumentBySymbol = {}
    for instrument in instruments:
        symbol = instrument["Symbol"]
        instrumentBySymbol[symbol] = instrument


    missingFields = []
    noneFields = []
    zerorFields = []
    #
    for field in fields:
        missingFields.append(0)
        noneFields.append(0)
        zerorFields.append(0)

    #
    step = 10000
    for i in range(13):
        filter = {"limit": step, "skip": i * step}
        totalFundamentals = database.Find("Stock", "Fundamental", filter)
        #
        for fundamental in totalFundamentals:
            symbol = fundamental["Symbol"]
            reportDate = fundamental["ReportDate"]
            instrument = instrumentBySymbol[symbol]
            w = fundamental["Wind"]

            #
            for i in range(len(fields)):
                field = fields[i]
                if field not in w:
                    missingFields[i] += 1
                    print(symbol, reportDate, field, "MissingField")
                elif w[field] == None:
                    noneFields[i] += 1
                    print(symbol, reportDate, field, "NoneField")
                elif w[field] == 0:
                    zerorFields[i] += 1
                    print(symbol, reportDate, field, "ZeroField")


    for i in range(len(fields)):
        field = fields[i]
        print(field, "#Missing", missingFields[i], "#None", noneFields[i], "#Zero", zerorFields[i])


def CheckNABar():
    count = database.Count("Stock", "DailyBar")
    batchSize = 10000
    batchCount = int(count / batchSize) + 1

    filter = {}
    filter["limit"] = batchSize
    #
    for i in range(batchCount):
        print(i)
        filter["skip"] = i * batchSize
        bars = database.Find("Stock", "DailyBar", filter)
        for bar in bars:
            symbol = bar["Symbol"]
            dt = bar["StdDateTime"]
            bClose = bar["Values"]["BClose"]
            if np.isnan(bClose):
                print(symbol, dt)
        #
        print("Loop", i, "of", batchCount, "Progress", round(i/batchCount,2)*100, "%", datetime.datetime.now())


# CheckNABar()

def CheckSalesAndTotalRevenue():

    #
    countSalesNone = 0
    countSalesZero = 0
    countRevenueZero = 0
    countNotEually = 0

    #
    step = 10000
    for i in range(13):
        filter = {"limit": step, "skip": i * step}
        # filter = {"Symbol": "000001.SZ"}

        totalFundamentals = database.Find("Stock", "Fundamental", filter)
        # print(i, len(totalFundamentals))

        for fundamental in totalFundamentals:
            symbol = fundamental["Symbol"]
            reportDate = fundamental["ReportDate"]
            w = fundamental["Wind"]
            v = fundamental["Values"]
            if "Sales" not in v:
                print(symbol, "Missing Sales")
                countSalesNone += 1
                continue
            if v["Sales"] == 0:
                countSalesZero += 1
            if v["TotalRevenue"] == 0:
                countRevenueZero += 1
            if v["Sales"] != v["TotalRevenue"]:
                countNotEually +=1
                print(symbol, reportDate)
    #
    print(countSalesNone, countSalesZero, countRevenueZero, countNotEually)

# CheckSalesAndTotalRevenue()


def CheckOperatingProfit():

    #filter = {"Symbol": "002755.SZ", "ReportDate": datetime.datetime(2017,12,31)}
    #fundamental = database.Find("Stock", "Fundamental", filter)
    #fundamental = fundamental[0]
    #wind = fundamental['Wind']
    #a1 = wind["net_invest_inc"]
    #a2 = wind["other_grants_inc"]
    #a3 = wind["net_gain_chg_fv"]
    #a4 = wind["gain_asset_dispositions"]

    #
    instruments = database.Find("Instruments", "Stock")
    instrumentBySymbol = {}
    for instrument in instruments:
        symbol = instrument["Symbol"]
        instrumentBySymbol[symbol] = instrument

    fields = ["opprofit","tot_oper_rev", "net_invest_inc", "other_grants_inc", "net_gain_chg_fv", "gain_asset_dispositions", "net_gain_fx_trans"]
    #
    missingFields = {}

    inCorrectOP = []
    inCorrectCount = 0
    #
    step = 10000
    for i in range(13):
        filter = {"limit":step, "skip":i * step}
        #filter = {"Symbol": "000001.SZ"}

        totalFundamentals = database.Find("Stock", "Fundamental", filter)
        # print(i, len(totalFundamentals))

        for fundamental in totalFundamentals:
            symbol = fundamental["Symbol"]
            reportDate = fundamental["ReportDate"]
            instrument = instrumentBySymbol[symbol]
            industry = instrument["Properties"]["SWIndustry1"]

            # print(symbol, reportDate)

            wind = fundamental["Wind"]
            completed = True
            fixNone = {}
            for field in fields:
                if field not in wind:
                    # print(symbol, reportDate, "no field", field)
                    completed = False
                    wind[field] = 0
                    if field not in missingFields:
                        missingFields[field] = []
                    missingFields[field].append([symbol, reportDate.date()])
                else:
                    if wind[field] == None:
                        fixNone[field] = 0
            #
            # if len(fixNone) > 0:
            #    target = {"Symbol": symbol, "ReportDate": reportDate}
            #    database.Update("Stock", "Fundamental", target, fixNone)
            # continue

            #if not completed:
            #    continue

            if industry == "银行" or industry == "非银行金融":
                continue
            else:
                pass

            if "opprofit" not in wind:
                print(symbol, reportDate, "No opprofit Field")
                continue

            if "tot_oper_rev" not in wind:
                print(symbol, reportDate, "No tot_oper_rev Field")
                continue

            if "tot_oper_cost" not in wind:
                print(symbol, reportDate, "No tot_oper_rev Field")
                continue

            fields2 = ["net_invest_inc", "other_grants_inc", "net_gain_chg_fv",
                      "gain_asset_dispositions", "net_gain_fx_trans"]


            a1 = wind["net_gain_fx_trans"]
            a2 = wind["net_invest_inc"]
            a3 = wind["other_grants_inc"]
            a4 = wind["net_gain_chg_fv"]
            a5 = wind["gain_asset_dispositions"]

            for field in fields:
                if field not in wind or wind[field] == None:
                    wind[field] = 0

            a6 = 0
            if "net_other_oper_inc" in wind:
                a6 = wind["net_other_oper_inc"]
            if a6 == None:
                a6 = 0

            op = fundamental["Wind"]["opprofit"]
            ext2 =  + wind["other_grants_inc"] \
                   + wind["net_gain_chg_fv"] \
                   + wind["gain_asset_dispositions"] \
                   + wind["net_gain_fx_trans"]
            ext1 = wind["non_oper_rev"] - wind["non_oper_exp"]

            interest = wind["stmnote_finexp_4"] - wind["stmnote_finexp_5"]
            op2 = op - ext2
            op1 = op2 + interest - wind["net_invest_inc"]

            op1forward = wind["tot_oper_rev"] + a6 - wind["tot_oper_cost"] + interest
            op2forward = wind["tot_oper_rev"] + a6 - wind["tot_oper_cost"] + wind["net_invest_inc"]


            if math.fabs(op1 - op1forward) > 0.01:
                diff = op1/op1forward-1
                print(symbol, reportDate, instrument["Description"], op1, op1forward, diff * 100)
                inCorrectOP.append([symbol, reportDate.date(), diff])
                inCorrectCount += 1
            pass
            #if "oper_cost" not in fundamental["Wind"]:
            #    print(symbol, reportDate, "No oper_cost Field")
            #elif fundamental["Wind"]["oper_cost"] == 0:
            #    print(symbol, reportDate, instrument["Description"], instrument["Properties"]["Industry"], fundamental["Wind"]["tot_oper_cost"])

        kk = 0

    print("InCorrect", inCorrectCount)

    # ---Export Incorrect---
    df = pd.DataFrame(inCorrectOP, columns=["Symbol", "ReportDate", "Diff"])
    df.to_csv("d:/data/fundamental/InCorrectOP-Adjusted2.csv")

    # ---Export missing fields---
    #for key, value in missingFields.items():
    #    data = []
    #    for v in value:
    #        data.append(v)
    #    df = pd.DataFrame(data, columns=["Symbol", "ReportDate"])
    #    df.to_csv("d:/data/fundamental/missingfield_" + str(key) + ".csv")

#CheckOperatingProfit()


def TansitDataAsWindFundamental():
    totalFundamental = database.Find("Stock", "Fundamental")

    for f in totalFundamental:
        doc = f["Values"]
        doc["ReportDate"] = f["ReportDate"]
        doc["Symbol"] = f["Symbol"]
        doc["ReleaseDate"] = f["ReleaseDate"]
        doc["DateTime"] = f["ReleaseDate"]


def TransitDataAsFundamental2():
    pass


def AddReportDateTimeToFundamentalDatabase(database):
    db = database.client["Fundamental"]
    fundamentalCollections = db.collection_names(False)
    for collectionName in fundamentalCollections:
        coll = database.client["Fundamental"][collectionName]
        docs = database.findAll("Fundamental", collectionName)
        for doc in docs:
            id = doc["_id"]
            update = {}
            reportDate = doc["ReportDate"]
            reportDateTime = Gadget.ParseDateTime(reportDate)
            reportDateTime = Gadget.ToUTCDateTime(reportDateTime)
            update["$set"] = {"StdReportDate": reportDateTime}
            coll.update_one({"_id": id}, update, False)


def AddOneExtraordinary(database, symbol, reportDate):
    fields = ["net_invest_inc", "other_grants_inc", "net_gain_chg_fv",
              "gain_asset_dispositions", "net_gain_fx_trans"]
    #
    DataProcess.Download_Fundamentals_Fields(database, symbol, reportDate, fields=fields, params={})


def AddExtraordinary():

    #
    fields = ["net_invest_inc", "other_grants_inc", "net_gain_chg_fv",
               "gain_asset_dispositions", "net_gain_fx_trans"]

    # fields = ["net_gain_chg_fv", "net_gain_fx_trans"]
    dfData = pd.read_csv("d:/data/fundamental/InCorrectOP1.csv")
    for index, row in dfData.iterrows():
        #
        if index < 28478:
            continue
        symbol = row["Symbol"]
        reportDate = row["ReportDate"]
        # reportDate = Gadget.ParseDateTime(reportDate)
        reportDate = datetime.datetime.strptime(reportDate, "%Y-%m-%d")
        print(index, symbol, reportDate)
        DataProcess.Download_Fundamentals_Fields(database, symbol, reportDate, fields=fields, params={})

# Wind.w.start()
# FixField()
# AddExtraordinary()

def AddOpratingProfit():

    step = 10000
    for i in range(13):
        filter = {"limit": step, "skip": i * step}
        # filter = {"Symbol": "000001.SZ"}

        totalFundamentals = database.Find("Stock", "Fundamental", filter)
        # print(i, len(totalFundamentals))

        for fundamental in totalFundamentals:
            symbol = fundamental["Symbol"]
            reportDate = fundamental["ReportDate"]
            wind = fundamental["Wind"]
            #
            doc = {}
            doc["Values.OperatingProfit"] = wind["opprofit"]
            target = {"Symbol": symbol, "ReportDate": reportDate}
            database.Update(target, "Fundamental", doc)


def AddOtherOperInc():
    #net_other_oper_inc

    #
    fields = ["net_other_oper_inc"]

    # fields = ["net_gain_chg_fv", "net_gain_fx_trans"]
    dfData = pd.read_csv("d:/data/fundamental/InCorrectOP-Adjusted.csv")
    for index, row in dfData.iterrows():
        #
        symbol = row["Symbol"]
        reportDate = row["ReportDate"]
        # reportDate = Gadget.ParseDateTime(reportDate)
        reportDate = datetime.datetime.strptime(reportDate, "%Y-%m-%d")

        if reportDate.year < 2018:
            print(index, symbol, reportDate)
            DataProcess.Download_Fundamentals_Fields(database, symbol, reportDate, fields=fields, params={})

# AddOtherOperInc()


def FixDateTimeAndReleaseDateTime(database):

    for i in range(13):
        filter = {"limit":10000, "skip":i * 10000}
        #filter = {"Symbol": "000001.SZ"}

        totalFundamentals = database.Find("Stock", "Fundamental", filter)
        print(i, len(totalFundamentals))

        #
        for fundamental in totalFundamentals:
            symbol = fundamental["Symbol"]
            datetimes = fundamental["DateTime"]

            #
            doc = {}
            if isinstance(datetimes, str):
                datetimesT = Gadget.ParseDateTime(datetimes)
                #datetimesT = Gadget.ToLocalDateTime(datetimesT)
                doc["DateTime"] = datetimesT
            else:
                datetimesT = datetimes

            #
            reportDate = fundamental["ReportDate"]
            releaseDate = fundamental["ReleaseDate"]
            if releaseDate.date() != datetimesT.date():
                print(symbol, datetimes)
                doc["ReleaseDate"] = datetimesT
            #
            target = {"Symbol": symbol, "ReportDate": reportDate}

            if len(doc) > 0:
                database.Update("Fundamental", "Stock", target)

        pass


# FixDateTimeAndReleaseDateTime(database)


def FixIndustry():
    fields = ["Industry"]
    fields.append("SWIndustry1")
    fields.append("SWIndustry2")
    fields.append("SWIndustry3")
    fields.append("CSIndustry1")
    fields.append("CSIndustry2")
    fields.append("CSIndustry3")

    #
    instruments = database.Find("Instruments","Stock")
    for instrument in instruments:
        #
        symbol = instrument["Symbol"]
        doc = {}
        for field in fields:
            industry = instrument["Properties"][field]
            if isinstance(industry, dict):
                print(instrument["Symbol"])
                doc[field] = instrument["Properties"][field]["industry2"]
            else:
                pass
                doc[field] = instrument["Properties"][field]
        #
        if len(doc):
            database.Upsert("Instruments","Stock", {"Symbol": symbol}, doc)
        k = 1
#FixIndustry()


def FixOperatingProfit2(fundamentals, instrumentBySymbol):

    for fundamental in fundamentals:
        symbol = fundamental["Symbol"]
        datetimes = fundamental["DateTime"]
        reportDate = fundamental["ReportDate"]
        wind = fundamental["Wind"]

        #
        if "opprofit" not in wind:
            print(symbol, reportDate, "No opprofit Field")
            continue

        if "tot_oper_rev" not in wind:
            print(symbol, reportDate, "No tot_oper_rev Field")
            continue

        #
        fields = ["net_invest_inc", "other_grants_inc", "net_gain_chg_fv",
                  "gain_asset_dispositions", "net_gain_fx_trans"]

        ext2 = 0
        for field in fields:
            if field not in wind:
                wind[field] = 0
            elif wind[field] == None:
                wind[field] = 0
            #
            ext2 += wind[field]

        ext1 = wind["non_oper_rev"] - wind["non_oper_exp"]

        #
        interest = wind["stmnote_finexp_4"] - wind["stmnote_finexp_5"]
        op = wind["opprofit"]

        instrument = instrumentBySymbol[symbol]
        industry = instrument["CSIndustry1"]

        if industry == "银行":
            op1 = wind["opprofit"]
        elif industry == "非银行金融":
            op1 = wind["opprofit"]
        else:
            op1forward = wind["tot_oper_rev"] - wind["tot_oper_cost"] + interest
            op2forward = wind["tot_oper_rev"] - wind["tot_oper_cost"] + wind["net_invest_inc"]
            op1 = op1forward

        # ---Operating Profit---
        doc = {}
        doc["Values.OperatingProfit"] = op
        doc["Values.OperatingProfit1"] = op1

        # ---Extraordinary---
        doc["Values.Extraordinary"] = ext1 + ext2

        #
        target = {"Symbol": symbol, "ReportDate": reportDate}

        if len(doc) > 0:
            database.Update(target, "Fundamental", doc)
            pass


def FixOperatingProfit():
    #
    instruments = database.Find("Instruments", "Stock")
    instrumentBySymbol = {}
    for instrument in instruments:
        symbol = instrument["Symbol"]
        instrumentBySymbol[symbol] = instrument
    #
    for i in range(13):
        filter = {"limit":10000, "skip":i * 10000}
        #filter = {"Symbol": "000001.SZ"}

        totalFundamentals = database.Find("Stock", "Fundamental", filter)
        print(i, len(totalFundamentals))

        #
        FixOperatingProfit2(totalFundamentals, instrumentBySymbol)
        pass


def FixOperatingProfitByReportDate(reportDate):
    #
    instruments = database.Find("Instruments", "Stock")
    instrumentBySymbol = {}
    for instrument in instruments:
        symbol = instrument["Symbol"]
        instrumentBySymbol[symbol] = instrument
    #
    filter = {"ReportDate": reportDate}
    totalFundamentals = database.Find("Stock", "Fundamental", filter)
    print(len(totalFundamentals))
    #
    FixOperatingProfit2(totalFundamentals, instrumentBySymbol)
    pass


def FixMissingFormattedValues():

    count = database.Count("Stock", "Fundamental")
    batchCount = int(count / 10000) + 1

    for i in range(batchCount):
        filter = {"limit": 10000, "skip": i * 10000}
        # filter = {"Symbol": "000001.SZ"}

        fundamentals = database.Find("Stock", "Fundamental", filter)
        print(i, len(fundamentals))
        count = 0
        #
        for fundamental in fundamentals:
            symbol = fundamental["Symbol"]
            datetimes = fundamental["DateTime"]
            reportDate = fundamental["ReportDate"]
            wind = fundamental["Wind"]
            values = fundamental["Values"]
            if len(wind) > 100 and len(values) < 50: # AndFix
                print(symbol, reportDate)

                # 以防万一，先补充字段
                AddOneExtraordinary(database, symbol, reportDate)

                # ---Fix---
                instrument = database.Find("Instruments", "Stock", {"Symbol": symbol})[0]
                DataProcess.FormatingReport(database, instrument, reportDate)

        pass


def FindMissingReports():

    missingReports = DataProcess.MissingReports()

    dfData = pd.read_csv("d:/data/fundamental/missingfield_opprofit.csv")
    for index, row in dfData.iterrows():
        #
        symbol = row["Symbol"]
        reportDate = row["ReportDate"]
        # print(reportDate)
        found = False

        i = 0
        for m in missingReports:
            reportDate2 = Gadget.ToDateString(m["ReportDate"])
            # print(reportDate2)

            if m["Symbol"] == symbol and reportDate2 ==reportDate:
                found = True
                break
            i += 1

        if found:
            missingReports.pop(i)

        if not found:
            print(symbol, reportDate, "Find New Missing Report")

    if len(missingReports) > 0:
        for m in missingReports:
            print(m["Symbol"], m["ReportDate"], "Old Missing Report Get Value")
#FindMissingReports()


def FixField():

    symbol = "002755.SZ"
    reportDate = datetime.datetime(2017,12,31)

    fields = ["net_invest_inc", "other_grants_inc", "net_gain_chg_fv",
              "gain_asset_dispositions", "net_gain_fx_trans"]
    #fields = ["gain_asset_dispositions"]

    instruments = database.Find("Instruments","Stock", {"Symbol": symbol})
    instrument = instruments[0]
    datetime2 = instrument["DateTime2"]
    if datetime.datetime.now().date() < instrument["DateTime2"].date():
        datetime2 = datetime.datetime.now()
        datetime2 = Gadget.ToUTCDateTime(datetime2)

    datetimes = Gadget.GenerateReportDates(instrument["DateTime1"], datetime2)
    for reportDate in datetimes:
        DataProcess.Download_Fundamentals_Fields(database, symbol, reportDate, fields=fields, params={})


def MoveToInside():
    step = 10000
    count1 = 0
    count2 = 0
    count3 = 0
    count4 = 0
    for i in range(13):
        filter = {"limit": step, "skip": i * step}
        # filter = {"Symbol": "000001.SZ"}

        totalFundamentals = database.Find("Stock", "Fundamental", filter)
        print(i, len(totalFundamentals))

        for fundamental in totalFundamentals:
            symbol = fundamental["Symbol"]
            reportDate = fundamental["ReportDate"]
            # instrument = instrumentBySymbol[symbol]

            if "gain_asset_dispositions" in fundamental:
                count1 += 1
                if fundamental["gain_asset_dispositions"] > 0:
                    count3 += 1
            if "other_grants_inc" in fundamental:
                count2 += 1
                if fundamental["other_grants_inc"] > 0:
                    count4 += 1


    print(count1,count3,count2,count4)


def FixInvestingCashFlow():
    #
    for i in range(13):
        filter = {"limit": 10000, "skip": i * 10000}
        # filter = {"Symbol": "000001.SZ"}

        fundamentals = database.Find("Stock", "Fundamental", filter)
        print(i, len(fundamentals))
        count = 0
        #
        for fundamental in fundamentals:
            symbol = fundamental["Symbol"]
            datetimes = fundamental["DateTime"]
            reportDate = fundamental["ReportDate"]
            wind = fundamental["Wind"]

            if "net_cash_flows_inv_act" not in wind:
                print(symbol, reportDate)
                count += 1
                continue

            # ---Operating Profit---
            doc = {}
            doc["Values.InvestingCashFlow"] = wind["net_cash_flows_inv_act"]
            #
            target = {"Symbol": symbol, "ReportDate": reportDate}

            if len(doc) > 0:
                database.Update(target, "Fundamental", doc)
                database.Update(target, "Fundamental", {"Values.InvstCashFlow": ""})
                pass

            #
    print("missing", count)


def FixDailyBarDateTime(insrumentType = "Stock"):

    count = database.Count(insrumentType, "DailyBar")
    batchSize = 10000
    batchCount = int(count / batchSize) + 1

    filter = {}
    filter["limit"] = batchSize
    #
    for i in range(batchCount):
        print(i)
        filter["skip"] = i * batchSize
        bars = database.Find(insrumentType, "DailyBar", filter)
        for bar in bars:
            symbol = bar["Symbol"]
            dt = bar["StdDateTime"]
            # bClose = bar["Values"]["BClose"]
            # if np.isnan(bClose):
            #    print(symbol, dt)

            document = {}

            localDateTime = Gadget.ToLocalDateTime(dt)
            localDateTime = Gadget.DeleteTimeZone(localDateTime)  # 必须删除TimeZone
            document["DateTime"] = localDateTime
            document["Date"] = Gadget.ToDate(localDateTime)
            #
            values = bar["Values"]
            for key, value in values.items():
                document[key] = value

            #
            target = {"Symbol": symbol, "StdDateTime": dt}
            if len(document) > 0:
                database.Update(target, "DailyBar", document)
        #
        print("Loop", i, "of", batchCount, "Progress", round(i / batchCount, 2) * 100, "%", datetime.datetime.now())




#database.creatIndex("Index","DailyBar","DateTime")
#database.creatIndex("Stock","DailyBar","DateTime")
# FixDailyBarDateTime(insrumentType="Index")

# MoveToInside()

#InvstCashFlow

# FixInvestingCashFlow()

# fields = ["opprofit","tot_oper_rev", "net_invest_inc", "other_grants_inc", "net_gain_chg_fv", "gain_asset_dispositions", "net_gain_fx_trans"]
# CheckNullField(["net_other_oper_inc"])
# CheckNullField(fields)
# FixOperatingProfit()
# FixOperatingProfitByReportDate(datetime.datetime(2019,3,31))

# Wind.w.start()
# FixMissingFormattedValues()

